{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np                       #矩阵操作\n",
    "import pandas as pd                     #引入\n",
    "from sklearn import preprocessing        #引入赌热编码，进行数据处理\n",
    "from pandas import DataFrame\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import log_loss     #采用logloss作为评价指标\n",
    "import matplotlib.pyplot as plt          #画图\n",
    "import seaborn as sns                    #画图\n",
    "import time\n",
    "\n",
    "#import list_utils\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from utils import *\n",
    "from FE_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1、进行数据下采样\n",
    "先从train数据中按正负样本1：r的比例进行采样，并将其存入本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#实现正负样本按照1：r的比例采样，并提供存储功能\n",
    "filename= 'D:/Jupyter/w000/train'\n",
    "save_name='D:/Down_sample_data.csv'\n",
    "##filename:原始数据地址+名称，\n",
    "#save_name：输出文件地址+名称\n",
    "#r:正负样本比例,默认负样本数是正样本的1\n",
    "#data_num：所需样本的总量\n",
    "def down_sample(filename,save_name,r=1,chunksize=100,data_positive_num=10000,save=False,return_=True):\n",
    "    positive_num=data_positive_num\n",
    "    negtive_num=data_positive_num*r\n",
    "    positive_n=0\n",
    "    negtive_n=0\n",
    "    datas=pd.read_csv(filename,chunksize=chunksize)\n",
    "    i=0\n",
    "#采集正负样本\n",
    "    for data in datas:\n",
    "        data_all=pd.read_csv(filename,nrows=1)\n",
    "        positive=data[data.click==1]\n",
    "        negtive=data[data.click==0]\n",
    "#采集正样本        \n",
    "        if positive_n<positive_num:\n",
    "            data_all=data_all.append(positive,ignore_index=True)\n",
    "            positive_n+=positive.shape[0]\n",
    "#采集负样本           \n",
    "        if negtive_n<negtive_num:\n",
    "            if negtive.shape[0]<=positive.shape[0]*r:\n",
    "                data_all=data_all.append(negtive,ignore_index=True)\n",
    "                negtive_n+=negtive.shape[0]\n",
    "            else:\n",
    "                n=int(positive.shape[0]*r)\n",
    "                data_all=data_all.append(negtive.head(n),ignore_index=True)\n",
    "                negtive_n+=n\n",
    "#存储数据\n",
    "        if save:\n",
    "            if i==0:\n",
    "                data_all.to_csv(save_name,header=True,index_label=False)\n",
    "            else:\n",
    "                data_all.to_csv(save_name,mode='a',header=False,index_label=False)            \n",
    "        i+=1\n",
    "        if (positive_n>=positive_num)&(negtive_n>=negtive_num):\n",
    "            break\n",
    "        \n",
    "        print('positive_n:'+str(positive_n),'  ','negtive_n:'+str(negtive_n))\n",
    "        del data\n",
    "         \n",
    "        \n",
    "    if return_:    \n",
    "        return data_all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $$$执行代码1：下采样\n",
    "#### 参数设置：\n",
    "filename：数据读取地址；save_name：下采样数据存储地址；r:正负样本比例；chunksize：单次处理数据，建议1w，这样下采样更均匀；data_positive_num：需采正样本数量，建议600w以下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive_n:1706    negtive_n:1706\n",
      "positive_n:3436    negtive_n:3436\n",
      "positive_n:5276    negtive_n:5276\n",
      "positive_n:7011    negtive_n:7011\n",
      "positive_n:8764    negtive_n:8764\n",
      "positive_n:10470    negtive_n:10470\n",
      "positive_n:12217    negtive_n:12217\n",
      "positive_n:13964    negtive_n:13964\n",
      "positive_n:15692    negtive_n:15692\n",
      "positive_n:17490    negtive_n:17490\n",
      "positive_n:19222    negtive_n:19222\n",
      "positive_n:20972    negtive_n:20972\n",
      "positive_n:22685    negtive_n:22685\n",
      "positive_n:24396    negtive_n:24396\n",
      "positive_n:26117    negtive_n:26117\n",
      "positive_n:27869    negtive_n:27869\n",
      "positive_n:29695    negtive_n:29695\n",
      "positive_n:31396    negtive_n:31396\n",
      "positive_n:33210    negtive_n:33210\n",
      "positive_n:34951    negtive_n:34951\n",
      "positive_n:36600    negtive_n:36600\n",
      "positive_n:38390    negtive_n:38390\n",
      "positive_n:40099    negtive_n:40099\n",
      "positive_n:41825    negtive_n:41825\n",
      "positive_n:43573    negtive_n:43573\n",
      "positive_n:45191    negtive_n:45191\n",
      "positive_n:46706    negtive_n:46706\n",
      "positive_n:48214    negtive_n:48214\n",
      "positive_n:49712    negtive_n:49712\n",
      "positive_n:51211    negtive_n:51211\n",
      "positive_n:52731    negtive_n:52731\n",
      "positive_n:54208    negtive_n:54208\n",
      "positive_n:55742    negtive_n:55742\n",
      "positive_n:57246    negtive_n:57246\n",
      "positive_n:58778    negtive_n:58778\n",
      "positive_n:60374    negtive_n:60374\n",
      "positive_n:61836    negtive_n:61836\n",
      "positive_n:63314    negtive_n:63314\n",
      "positive_n:64859    negtive_n:64859\n",
      "positive_n:66356    negtive_n:66356\n",
      "positive_n:67878    negtive_n:67878\n",
      "positive_n:69359    negtive_n:69359\n",
      "positive_n:70844    negtive_n:70844\n",
      "positive_n:72317    negtive_n:72317\n",
      "positive_n:73805    negtive_n:73805\n",
      "positive_n:75368    negtive_n:75368\n",
      "positive_n:76985    negtive_n:76985\n",
      "positive_n:78660    negtive_n:78660\n",
      "positive_n:80331    negtive_n:80331\n",
      "positive_n:82037    negtive_n:82037\n",
      "positive_n:83767    negtive_n:83767\n",
      "positive_n:85439    negtive_n:85439\n",
      "positive_n:87143    negtive_n:87143\n",
      "positive_n:88868    negtive_n:88868\n",
      "positive_n:90548    negtive_n:90548\n",
      "positive_n:92228    negtive_n:92228\n",
      "positive_n:93887    negtive_n:93887\n",
      "positive_n:95598    negtive_n:95598\n",
      "positive_n:97285    negtive_n:97285\n",
      "positive_n:98979    negtive_n:98979\n",
      "positive_n:100734    negtive_n:100734\n",
      "positive_n:102454    negtive_n:102454\n",
      "positive_n:104173    negtive_n:104173\n",
      "positive_n:105839    negtive_n:105839\n",
      "positive_n:107574    negtive_n:107574\n",
      "positive_n:109170    negtive_n:109170\n",
      "positive_n:110692    negtive_n:110692\n",
      "positive_n:112174    negtive_n:112174\n",
      "positive_n:113643    negtive_n:113643\n",
      "positive_n:115114    negtive_n:115114\n",
      "positive_n:116651    negtive_n:116651\n",
      "positive_n:118176    negtive_n:118176\n",
      "positive_n:119675    negtive_n:119675\n",
      "positive_n:121158    negtive_n:121158\n",
      "positive_n:122635    negtive_n:122635\n",
      "positive_n:124210    negtive_n:124210\n",
      "positive_n:125662    negtive_n:125662\n",
      "positive_n:127135    negtive_n:127135\n",
      "positive_n:128722    negtive_n:128722\n",
      "positive_n:130256    negtive_n:130256\n",
      "positive_n:131743    negtive_n:131743\n",
      "positive_n:133329    negtive_n:133329\n",
      "positive_n:134862    negtive_n:134862\n",
      "positive_n:136393    negtive_n:136393\n",
      "positive_n:137867    negtive_n:137867\n",
      "positive_n:139387    negtive_n:139387\n",
      "positive_n:140956    negtive_n:140956\n",
      "positive_n:142521    negtive_n:142521\n",
      "positive_n:144033    negtive_n:144033\n",
      "positive_n:145531    negtive_n:145531\n",
      "positive_n:147063    negtive_n:147063\n",
      "positive_n:148497    negtive_n:148497\n",
      "positive_n:149970    negtive_n:149970\n",
      "positive_n:151416    negtive_n:151416\n",
      "positive_n:152810    negtive_n:152810\n",
      "positive_n:154279    negtive_n:154279\n",
      "positive_n:155741    negtive_n:155741\n",
      "positive_n:157200    negtive_n:157200\n",
      "positive_n:158716    negtive_n:158716\n",
      "positive_n:160219    negtive_n:160219\n",
      "positive_n:161710    negtive_n:161710\n",
      "positive_n:163203    negtive_n:163203\n",
      "positive_n:164774    negtive_n:164774\n",
      "positive_n:166289    negtive_n:166289\n",
      "positive_n:167711    negtive_n:167711\n",
      "positive_n:169224    negtive_n:169224\n",
      "positive_n:170713    negtive_n:170713\n",
      "positive_n:172216    negtive_n:172216\n",
      "positive_n:173710    negtive_n:173710\n",
      "positive_n:175236    negtive_n:175236\n",
      "positive_n:176752    negtive_n:176752\n",
      "positive_n:178211    negtive_n:178211\n",
      "positive_n:179681    negtive_n:179681\n",
      "positive_n:181123    negtive_n:181123\n",
      "positive_n:182639    negtive_n:182639\n",
      "positive_n:184137    negtive_n:184137\n",
      "positive_n:185614    negtive_n:185614\n",
      "positive_n:187119    negtive_n:187119\n",
      "positive_n:188636    negtive_n:188636\n",
      "positive_n:190180    negtive_n:190180\n",
      "positive_n:191813    negtive_n:191813\n",
      "positive_n:193344    negtive_n:193344\n",
      "positive_n:194946    negtive_n:194946\n",
      "positive_n:196562    negtive_n:196562\n",
      "positive_n:198216    negtive_n:198216\n",
      "positive_n:199750    negtive_n:199750\n",
      "positive_n:201334    negtive_n:201334\n",
      "positive_n:202909    negtive_n:202909\n",
      "positive_n:204581    negtive_n:204581\n",
      "positive_n:206178    negtive_n:206178\n",
      "positive_n:207839    negtive_n:207839\n",
      "positive_n:209511    negtive_n:209511\n",
      "positive_n:211232    negtive_n:211232\n",
      "positive_n:212871    negtive_n:212871\n",
      "positive_n:214494    negtive_n:214494\n",
      "positive_n:216098    negtive_n:216098\n",
      "positive_n:217693    negtive_n:217693\n",
      "positive_n:219322    negtive_n:219322\n",
      "positive_n:220891    negtive_n:220891\n",
      "positive_n:222451    negtive_n:222451\n",
      "positive_n:224077    negtive_n:224077\n",
      "positive_n:225698    negtive_n:225698\n",
      "positive_n:227289    negtive_n:227289\n",
      "positive_n:228922    negtive_n:228922\n",
      "positive_n:230631    negtive_n:230631\n",
      "positive_n:232412    negtive_n:232412\n",
      "positive_n:234117    negtive_n:234117\n",
      "positive_n:235812    negtive_n:235812\n",
      "positive_n:237412    negtive_n:237412\n",
      "positive_n:239084    negtive_n:239084\n",
      "positive_n:240816    negtive_n:240816\n",
      "positive_n:242488    negtive_n:242488\n",
      "positive_n:244264    negtive_n:244264\n",
      "positive_n:245978    negtive_n:245978\n",
      "positive_n:247700    negtive_n:247700\n",
      "positive_n:249370    negtive_n:249370\n",
      "positive_n:251025    negtive_n:251025\n",
      "positive_n:252734    negtive_n:252734\n",
      "positive_n:254461    negtive_n:254461\n",
      "positive_n:256190    negtive_n:256190\n",
      "positive_n:257899    negtive_n:257899\n",
      "positive_n:259590    negtive_n:259590\n",
      "positive_n:261290    negtive_n:261290\n",
      "positive_n:262957    negtive_n:262957\n",
      "positive_n:264622    negtive_n:264622\n",
      "positive_n:266325    negtive_n:266325\n",
      "positive_n:268043    negtive_n:268043\n",
      "positive_n:269729    negtive_n:269729\n",
      "positive_n:271439    negtive_n:271439\n",
      "positive_n:273123    negtive_n:273123\n",
      "positive_n:274828    negtive_n:274828\n",
      "positive_n:276488    negtive_n:276488\n",
      "positive_n:278221    negtive_n:278221\n",
      "positive_n:279870    negtive_n:279870\n",
      "positive_n:281532    negtive_n:281532\n",
      "positive_n:283257    negtive_n:283257\n",
      "positive_n:284887    negtive_n:284887\n",
      "positive_n:286638    negtive_n:286638\n",
      "positive_n:288274    negtive_n:288274\n",
      "positive_n:290046    negtive_n:290046\n",
      "positive_n:291788    negtive_n:291788\n",
      "positive_n:293453    negtive_n:293453\n",
      "positive_n:295131    negtive_n:295131\n",
      "positive_n:296874    negtive_n:296874\n",
      "positive_n:298578    negtive_n:298578\n",
      "positive_n:300243    negtive_n:300243\n",
      "positive_n:301905    negtive_n:301905\n",
      "positive_n:303538    negtive_n:303538\n",
      "positive_n:305188    negtive_n:305188\n",
      "positive_n:306862    negtive_n:306862\n",
      "positive_n:308505    negtive_n:308505\n",
      "positive_n:310165    negtive_n:310165\n",
      "positive_n:311764    negtive_n:311764\n",
      "positive_n:313375    negtive_n:313375\n",
      "positive_n:315078    negtive_n:315078\n",
      "positive_n:316699    negtive_n:316699\n",
      "positive_n:318314    negtive_n:318314\n",
      "positive_n:319984    negtive_n:319984\n",
      "positive_n:321620    negtive_n:321620\n",
      "positive_n:323209    negtive_n:323209\n",
      "positive_n:324913    negtive_n:324913\n",
      "positive_n:326496    negtive_n:326496\n",
      "positive_n:328172    negtive_n:328172\n",
      "positive_n:329722    negtive_n:329722\n",
      "positive_n:331322    negtive_n:331322\n",
      "positive_n:332994    negtive_n:332994\n",
      "positive_n:334620    negtive_n:334620\n",
      "positive_n:336254    negtive_n:336254\n",
      "positive_n:337952    negtive_n:337952\n",
      "positive_n:339710    negtive_n:339710\n",
      "positive_n:341563    negtive_n:341563\n",
      "positive_n:343278    negtive_n:343278\n",
      "positive_n:345101    negtive_n:345101\n",
      "positive_n:346875    negtive_n:346875\n",
      "positive_n:348624    negtive_n:348624\n",
      "positive_n:350438    negtive_n:350438\n",
      "positive_n:352270    negtive_n:352270\n",
      "positive_n:354093    negtive_n:354093\n",
      "positive_n:355822    negtive_n:355822\n",
      "positive_n:357621    negtive_n:357621\n",
      "positive_n:359407    negtive_n:359407\n",
      "positive_n:361201    negtive_n:361201\n",
      "positive_n:362995    negtive_n:362995\n",
      "positive_n:364785    negtive_n:364785\n",
      "positive_n:366545    negtive_n:366545\n",
      "positive_n:368334    negtive_n:368334\n",
      "positive_n:370160    negtive_n:370160\n",
      "positive_n:371910    negtive_n:371910\n",
      "positive_n:373879    negtive_n:373879\n",
      "positive_n:375844    negtive_n:375844\n",
      "positive_n:377918    negtive_n:377918\n",
      "positive_n:379860    negtive_n:379860\n",
      "positive_n:381891    negtive_n:381891\n",
      "positive_n:383887    negtive_n:383887\n",
      "positive_n:385984    negtive_n:385984\n",
      "positive_n:387971    negtive_n:387971\n",
      "positive_n:389941    negtive_n:389941\n",
      "positive_n:391945    negtive_n:391945\n",
      "positive_n:393969    negtive_n:393969\n",
      "positive_n:396026    negtive_n:396026\n",
      "positive_n:398000    negtive_n:398000\n",
      "positive_n:400052    negtive_n:400052\n",
      "positive_n:402135    negtive_n:402135\n",
      "positive_n:404114    negtive_n:404114\n",
      "positive_n:406156    negtive_n:406156\n",
      "positive_n:408213    negtive_n:408213\n",
      "positive_n:410177    negtive_n:410177\n",
      "positive_n:412224    negtive_n:412224\n",
      "positive_n:414180    negtive_n:414180\n",
      "positive_n:416185    negtive_n:416185\n",
      "positive_n:418132    negtive_n:418132\n",
      "positive_n:420086    negtive_n:420086\n",
      "positive_n:422097    negtive_n:422097\n",
      "positive_n:424078    negtive_n:424078\n",
      "positive_n:426063    negtive_n:426063\n",
      "positive_n:428114    negtive_n:428114\n",
      "positive_n:430130    negtive_n:430130\n",
      "positive_n:432148    negtive_n:432148\n",
      "positive_n:434167    negtive_n:434167\n",
      "positive_n:436234    negtive_n:436234\n",
      "positive_n:438157    negtive_n:438157\n",
      "positive_n:440035    negtive_n:440035\n",
      "positive_n:441845    negtive_n:441845\n",
      "positive_n:443667    negtive_n:443667\n",
      "positive_n:445521    negtive_n:445521\n",
      "positive_n:447343    negtive_n:447343\n",
      "positive_n:449088    negtive_n:449088\n",
      "positive_n:450931    negtive_n:450931\n",
      "positive_n:452762    negtive_n:452762\n",
      "positive_n:454640    negtive_n:454640\n",
      "positive_n:456510    negtive_n:456510\n",
      "positive_n:458352    negtive_n:458352\n",
      "positive_n:460284    negtive_n:460284\n",
      "positive_n:462139    negtive_n:462139\n",
      "positive_n:464041    negtive_n:464041\n",
      "positive_n:465938    negtive_n:465938\n",
      "positive_n:467882    negtive_n:467882\n",
      "positive_n:469720    negtive_n:469720\n",
      "positive_n:471595    negtive_n:471595\n",
      "positive_n:473645    negtive_n:473645\n",
      "positive_n:475579    negtive_n:475579\n",
      "positive_n:477577    negtive_n:477577\n",
      "positive_n:479599    negtive_n:479599\n",
      "positive_n:481614    negtive_n:481614\n",
      "positive_n:483666    negtive_n:483666\n",
      "positive_n:485629    negtive_n:485629\n",
      "positive_n:487716    negtive_n:487716\n",
      "positive_n:489770    negtive_n:489770\n",
      "positive_n:491790    negtive_n:491790\n",
      "positive_n:493801    negtive_n:493801\n",
      "positive_n:495757    negtive_n:495757\n",
      "positive_n:497852    negtive_n:497852\n",
      "positive_n:499843    negtive_n:499843\n"
     ]
    }
   ],
   "source": [
    "#实现正负样本按照1：r的比例采样，并提供存储功能\n",
    "filename= 'D:/Jupyter/w000/train'            #数据读取地址\n",
    "save_name='D:/Down_sample_data.csv'          #下采样数据存储地址\n",
    "down_sample(filename,save_name,r=1,chunksize=10000,data_positive_num=500000,save=True ,return_=False)\n",
    "#r:正负样本比例\n",
    "#chunksize：单次处理数据，建议1w，这样下采样更均匀\n",
    "#data_positive_num：需采正样本数量，建议600w以下"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2、特征工程预处理\n",
    "#### 2.1、增加用户识别特征 （用于生成新增的7个特征）\n",
    "通过将device_id+device_model作为用户的唯一识别码，因为联合后在训练、测试数据中的维度较device_id本身仅增加了2.8%。同时删除device_id和device_ip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $$$执行代码2：  增加用户识别码use_id，删除训练device_id和device_ip\n",
    "##### 参数设置：\n",
    "filename：数据读取地址；save_name：数据存储地址；chunksize：每次提取多少数据进行处理，取决于电脑内存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filename= 'D:/Down_sample_data.csv'\n",
    "save_name='D:/Jupyter/w000/train_23.csv'\n",
    "\n",
    "data_all=pd.read_csv(filename,chunksize=1000000)\n",
    "\n",
    "i=0\n",
    "for data in data_all:\n",
    "    data['user_id']=data['device_id']+data['device_model']\n",
    "    data.drop(['device_id','device_ip'],inplace=True,axis=1)\n",
    "    if i==0:\n",
    "        data.to_csv(save_name,header=True,index_label=False)\n",
    "    else  :\n",
    "        data.to_csv(save_name,mode='a',header=False,index_label=False)\n",
    "    \n",
    "    \n",
    "    del data\n",
    "    i+=1\n",
    "    \n",
    "        \n",
    "    print('Num_data:'+str(i*2000000))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2、原19维特征+hour展开3个特征的特征工程\n",
    " 由于pandas用replace处理大规模数据，非常占内存，故而对获取下采样数据进行分批次读取处理。\n",
    "  其中hour_days特征的增加用于生成新增的8个特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tip:由于使用特征工程时，replace操作很占内存，故而请使用2.2.2代码分批次读取数据，并将数据存入指定文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.1、 原始特征工程，生成非onehot编码\n",
    "      1、实现将除id、click、use_id外取其前1%特征维度的代码\n",
    "      2、实现将hour生成额外3种特征的代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对输入数据进行特征工程\n",
    "#data：需要进行特征工程的数据；\n",
    "#name_dir：特征工程所需的dict共两个，19维一个，时间一个\n",
    "def Feature_Engineering(data_,name_dir='D:/'):\n",
    "#除hour、id、click、device_id、device_ip外19维特征处理\n",
    "    start=time.time()\n",
    "    FE_19_=read_double_dict(name_dir+'FE_19.txt',key_int=False)\n",
    "#此操作：由txt读取的dict中的数据类型是str，故而需将其还原为int类型的数据\n",
    "    list_int=['click','hour','C1','banner_pos','device_type','device_conn_type','C14','C15','C16','C17','C18','C19','C20','C21']\n",
    "    FE_19={}\n",
    "    for k,v in FE_19_.items():\n",
    "        dict_={}\n",
    "        if k in list_int:\n",
    "            for key,value in v.items():\n",
    "                dict_[int(key)]=value\n",
    "        else:\n",
    "            for key,value in v.items():\n",
    "                dict_[key]=value        \n",
    "        FE_19[k]=dict_\n",
    "    \n",
    "    for key in FE_19.keys():\n",
    "        data_[key].replace(FE_19[key],inplace=True)\n",
    "        \n",
    "#——————————————————————————————————————————————        \n",
    "    FE_hour=read_double_dict(name_dir+'FE_hour.txt')\n",
    "#就hour增加哪个小时、星期几，第几天三类特征    \n",
    "    data_['hour_hours']=data_.hour.replace(FE_hour['FE_hour_hours'])\n",
    "    data_['hour_weekday']=data_.hour.replace(FE_hour['FE_hour_weekday'])\n",
    "    data_['hour_days']=data_.hour.replace(FE_hour['FE_hour_days'])\n",
    "    \n",
    "    end=time.time()\n",
    "    print(\"times\"+str(end-start))\n",
    "    return data_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.2、 分批次读取要进行特征工程的数据并将其存入一个文件中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "##由于采用pandas自带的replace所消耗的内存很大，故而需要分批处理数据并将其存到一个文件中。\n",
    "#filename：需特征工程的数据\n",
    "#save_name：文件保存地址\n",
    "#chunksize：每次对多少数据进行特征工程\n",
    "#data_num：总共需要多少完成特征工程的数据\n",
    "filename= 'D:/Jupyter/w000/train'\n",
    "save_name='D:/FE_data.csv'\n",
    "def FE_data(filename,save_name,chunksize=100000,data_num=10000000):\n",
    "    step_num=data_num//chunksize\n",
    "    print(step_num)\n",
    "    datas=pd.read_csv(filename,chunksize=chunksize)\n",
    "    i=0\n",
    "    for data in datas:\n",
    "        data_=Feature_Engineering(data)\n",
    "        if i==0:\n",
    "            data.to_csv(save_name,header=True,index_label=False)\n",
    "        else:\n",
    "            data.to_csv(save_name,mode='a',header=False,index_label=False)\n",
    "        i+=1\n",
    "        if i>=step_num:\n",
    "            break\n",
    "        del data\n",
    "    return data_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $$$执行代码3： 对前99%的数据进行数字编码，剩余1%编码为-1；将时间hour展开为3个新特征\n",
    "#### 参数设置：\n",
    "filename：数据读取地址；save_name：数据存储地址，chunksize：每批次处理数据数，建议10w；data_num：总共需处理数据数\n",
    "预计1h处理300w数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "times124.32160782814026\n",
      "times124.78383755683899\n",
      "times125.25509905815125\n",
      "times123.8774254322052\n",
      "times125.44318675994873\n",
      "times126.22883939743042\n",
      "times127.39102077484131\n",
      "times125.69707226753235\n",
      "times126.47747945785522\n",
      "times127.31162023544312\n"
     ]
    }
   ],
   "source": [
    "filename= 'D:/Jupyter/w000/train_23.csv'\n",
    "save_name='D:/Jupyter/w000/FE_data_train_26.csv'\n",
    "data_FE=FE_data(filename,save_name,chunksize=100000,data_num=5000000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 进行新增的8个特征工程处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1、user_hour_C14：每个用户在小时内查看该广告的次数\n",
    "##### 2、user_hour_C17：每个用户，在小时内查看该广告告的次数\n",
    "##### 3、user_day_C14：每个用户，在当天查看该广告的次数\n",
    "##### 4、user_day_C17：每个用户，在当天查看该类型广告的次数\n",
    "##### 5、user_day_times:每个用户，当天的出现次数\n",
    "##### 6、user_day_app_id：每个用户，一天在此app_id中登录的次数\n",
    "##### 7、user_days:每个用户，累计登陆的天数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "此块代码耗时非常长，原因是先用bygroup生成所需用户热度(频率)的dataframe，再遍历数据根据关键字从dataframe提取对应频率。所处理数据越多\n",
    "#生成频率的dataframe就越大，在遍历数据查找所消耗的时间就越多。    经测算，600w数据，用单核预计得9天。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $$$执行代码4： 生成新的7个表示热度的特征，同时删除'id','hour','user_id'这三个特征\n",
    "### $请依次执行以下4块代码\n",
    "#### 参数设置：\n",
    "filename：数据读取地址；save_name：生成数据存储地址"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename= 'D:/Jupyter/w000/FE_data_train_26.csv'\n",
    "save_name='D:/Jupyter/w000/FE_data_train_30.csv'\n",
    "data=pd.read_csv(filename,nrows=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#1、根据user_id,hour,C14生成用户在每个小时段，登陆C14的次数表格\n",
    "user_hour_C14_dict=data.groupby(['user_id', 'hour','C14'],as_index=False,squeeze=True).size().reset_index()\n",
    "#2、根据user_id,hour,C17生成用户在每个小时段，登陆C17的次数表格\n",
    "user_hour_C17_dict=data.groupby(['user_id', 'hour','C17'],as_index=False,squeeze=True).size().reset_index()\n",
    "#3、根据user_id,hour_days,C14生成用户在每天，登陆C14的次数表格\n",
    "user_day_C14_dict=data.groupby(['user_id', 'hour_days','C14'],as_index=False,squeeze=True).size().reset_index()\n",
    "#4、根据user_id,hour_days,C17生成用户在每天，登陆C14的次数表格\n",
    "user_day_C17_dict=data.groupby(['user_id', 'hour_days','C17'],as_index=False,squeeze=True).size().reset_index()\n",
    "\n",
    "#5、根据user_id,hour_days生成用户在每天出现次数表格\n",
    "user_day_times_dict=data.groupby(['user_id', 'hour_days'],as_index=False,squeeze=True).size().reset_index()\n",
    "\n",
    "#6、根据user_id,hour_days，app_id生成用户每天登陆特定app的次数\n",
    "user_day_app_id_dict=data.groupby(['user_id', 'hour_days','app_id'],as_index=False,squeeze=True).size().reset_index()\n",
    "\n",
    "#7、在user_day_times_dict的基础上对user_id进行排序，得到每个用户出现的天数\n",
    "user_days_dict=user_day_times_dict.groupby(['user_id'],as_index=False,squeeze=True).size().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 生成7个新特征数据\n",
    "##### tip：10w数据差不多消耗20min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6.26265287399292\n"
     ]
    }
   ],
   "source": [
    "#相关find_user_hour_C14函数在FE_utils\n",
    "start=time.time()\n",
    "data['user_hour_C14'] = data.apply(lambda row: find_user_hour_C14(row['user_id'], row['hour'], row['C14'],user_hour_C14_dict), axis=1)\n",
    "data['user_hour_C17'] = data.apply(lambda row: find_user_hour_C17(row['user_id'], row['hour'], row['C17'],user_hour_C17_dict), axis=1)\n",
    "data['user_day_C14'] = data.apply(lambda row: find_user_day_C14(row['user_id'], row['hour_days'], row['C14'],user_day_C14_dict), axis=1)\n",
    "data['user_day_C17'] = data.apply(lambda row: find_user_day_C17(row['user_id'], row['hour_days'], row['C17'],user_day_C17_dict), axis=1)\n",
    "\n",
    "data['user_day_times'] = data.apply(lambda row: find_user_day_times(row['user_id'], row['hour_days'],user_day_times_dict), axis=1)\n",
    "data['user_day_app_id'] = data.apply(lambda row: find_user_day_app_id(row['user_id'], row['hour_days'], row['app_id'],user_day_app_id_dict), axis=1)\n",
    "\n",
    "data['user_days'] = data.apply(lambda row: find_user_days(row['user_id'],user_days_dict), axis=1)\n",
    "end=time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#由于数据此前经历过下采样，正负样本相对集中，故而此处将数据的\n",
    "data.sample(frac=1).reset_index(drop=True)\n",
    "#将id、hour、user_id三个特征删除\n",
    "data.drop(['id','hour','user_id'],inplace=True,axis=1)\n",
    "#在删除了'id','hour','user_id'后，共有特征30个。\n",
    "#其中类别型特征21个\n",
    "#数值型特征共8个：hour_days、user_hour_C14、user_hour_C17、user_day_C14、user_day_C17、user_day_times、user_day_app_id、user_days\n",
    "data.to_csv(save_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
